package com.spider.ask120.webmagic;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import com.spider.utils.HttpClientUtil;


/**
 * 描        述：爬数据-健康咨询(频道)
 * 创建时间：2016-8-15
 * @author Jibaole
 */
public class Demo implements PageProcessor {
	   public static final String URL_INDEX = "http://m.120ask.com/healthart/wx_art_home_all/";
	   public static final String URL_LIST = "/?kswys_wx=1&from=atr_home_all";
	   public static final String URL_tag_list = "/healthart/wx_lists/\\?tid=[0-9]+";
	   public static final String URL_tag_list_cont = "/cron/wxartlist/jump";
	
	// 部分一：抓取网站的相关配置，包括编码、抓取间隔、重试次数等
	private Site site = Site.me().setRetryTimes(3).setSleepTime(1);
	public Site getSite() {  
		return site;
	}

	 @Override
	    public void process(Page page) {
	        //列表页
	        if (page.getUrl().regex(URL_INDEX+"$").match()) {
	        	System.err.println("#########################>>>频道页<<<<###################################");
	        	//获取频道列表URL 
	        	List<String> list = page.getHtml().xpath("//ul[@class='ChanelList']/a[@href]/@href").all(); 
	        	page.putField("title", page.getHtml().xpath("//ul[@class='ChanelList']/a/li/p/b/text()"));
	        	page.putField("jianjie", page.getHtml().xpath("//ul[@class='ChanelList']/a/li/p/span/text()"));
	        	page.putField("img", page.getHtml().xpath("//ul[@class='ChanelList']/a/li/img/@src"));
	        	list.remove(2);//移除健康饮食
	        	list.remove(2);//移除民间偏方
	        	list.remove(3);//移除健康经验
	        	List<String> list1= new ArrayList<String>();
	        	list1.add(list.get(0));
	        	page.addTargetRequests(list1);
	            //tag 标签
	        } else if(page.getUrl().regex(URL_LIST+"$").match()) {
	        	System.err.println("#########################>>>标签页<<<<###################################");
	        	//得到所有标签
	        	page.putField("tag", page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li/a/text()").all());
	        	List<String> tag = page.getHtml().xpath("//div[@class='ChanelHomeBox']/ul/li/a/@href").all();
	        	List<String> list1= new ArrayList<String>();
	        	list1.add(tag.get(0));
	        	page.addTargetRequests(list1);
	        //tag下，列表页
	        }else if(page.getUrl().regex(URL_tag_list).match()){
	        	System.err.println("#########################>>>详细内容，列表页<<<<###################################");
	        	List<String> urlList = new ArrayList<String>();
	        	String url = page.getUrl().toString();
	        	String tid=url.substring(url.indexOf("=")+1, url.indexOf("&"));
					//无下级栏目，开始加载分页数据列表，获取正文URL
					Map<String, String> params = new HashMap<String, String>();
					params.put("tid",tid);
					params.put("tj_type", "5");
					params.put("order", "1");
					int pageIndex = 0;	
					while(true){ 
						pageIndex ++;
						params.put("page", pageIndex+"");
						System.out.println("第"+pageIndex+"页");
						String listJson = HttpClientUtil.getInstance().sendHttpPost("http://m.120ask.com/healthart/wx_art_more/", params);
						if(!StringUtils.isEmpty(listJson)){
								try {
									Document doc = Jsoup.parse(listJson);
									//遍历标签
									for (Element link : doc.getElementsByAttribute("href")) {
									 String linkUrl = link.attr("href");
									 page.putField("tag_list_url",linkUrl);
									 page.putField("tag_list_img",link.select("img").attr("src"));
									 page.putField("tag_list_title",link.select("dd").get(0).text());
									 page.putField("tag_list_word",link.select("dd").get(1).text());
									 System.out.println("内容列表Url:"+linkUrl);
									 urlList.add(linkUrl);
									}
							} catch (Exception e) { 
								e.printStackTrace();
							}
						}else{
							break;
						}
						page.addTargetRequests(urlList);
					}
	        //详细内容页
	        }else if(page.getUrl().toString().indexOf(URL_tag_list_cont) > 0){
	        	System.err.println("#########################>>>详细内容页<<<<###################################");
	        	page.putField("cont_title", page.getHtml().xpath("//div[@class='article_cont']/dl/dd/text()")); 
	        	page.putField("cont_img", page.getHtml().xpath("//div[@class='article_cont-div']/span/img/@data-original"));
	        	page.putField("cont_text", page.getHtml().xpath("//div[@class='article_cont-divp']/html()"));
	        	String cont = page.getHtml().xpath("//dd[@class='article_cont-dd2']/html()").toString();
	        	String create_time = page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[1]/text()").toString();
	        	String  browse_count = page.getHtml().xpath("//dd[@class='article_cont-dd2']/span[2]").toString();
	        	System.out.println("create_time="+create_time+"browse_count:"+browse_count);
	        	if(!StringUtils.isEmpty(browse_count)){
	        		if(browse_count.indexOf("(") >0 && browse_count.indexOf(")") >0){
	        			System.out.println("browse_count:"+browse_count.substring(browse_count.indexOf("(")+1, browse_count.indexOf(")")));
	        		}
	        	}
	        	System.out.println(browse_count);
	        }
	 }
	 
	public static void main(String[] args) {
		Spider.create(new Demo()).addUrl("http://m.120ask.com/healthart/wx_art_home_all/").thread(1).run(); 
    }
 
}
